package com.djhu.newscollector.crawel.imp.cl;

import com.djhu.newscollector.core.Holder.JobHolder;
import com.djhu.newscollector.core.dispatch.JobDispatcher;
import com.djhu.newscollector.core.entity.Response;
import com.djhu.newscollector.crawel.core.PageProcess;
import com.djhu.newscollector.crawel.core.Url;
import com.djhu.newscollector.crawel.imp.worker.NewsResponse;
import com.djhu.newscollector.crawel.imp.worker.WangyiNewsWorker;
import com.djhu.newscollector.crawel.net.DefaultHttpVisitor;
import com.djhu.newscollector.crawel.net.HttpRequest;
import com.djhu.newscollector.crawel.net.HttpVisitor;
import com.djhu.newscollector.crawel.net.Page;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.util.Set;

/**
 * Created by zw on 2018/2/12.
 */
public class IndexPageProcess implements PageProcess {
    private JobHolder jobHolder;
    private JobDispatcher jobDispatcher;
    private static Logger logger = LogManager.getLogger(IndexPageProcess.class.getName());

    @Override
    public Response process(Url url) {


        HttpRequest request = new HttpRequest();
        request.setUrl(url);
        HttpVisitor httpVisitor = new DefaultHttpVisitor();
        Page page = null;
        try {
            page = httpVisitor.visit(request);
            Set<String> urls = page.getLinks();
            for (String link:urls) {
                if (!link.contains("htm_data")) {
                    continue;
                }
                Url linkUrl = new Url();
                linkUrl.setUrl(link);
                if (jobHolder.contains(linkUrl)) {
                    logger.info("link has craweled!!!" + link);
                    continue;
                } else {
                    jobDispatcher.retJob(linkUrl);
                }
            }
        }catch (Exception e){
        }
        NewsResponse newsResponse = new NewsResponse();
        newsResponse.setPage(page);
        return newsResponse;
    }

    public JobHolder getJobHolder() {
        return jobHolder;
    }

    public void setJobHolder(JobHolder jobHolder) {
        this.jobHolder = jobHolder;
    }

    public JobDispatcher getJobDispatcher() {
        return jobDispatcher;
    }

    public void setJobDispatcher(JobDispatcher jobDispatcher) {
        this.jobDispatcher = jobDispatcher;
    }
}